# coding:utf-8
import re
import os
import time
import uuid
import pymysql
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup
from string import Template
from pymongo import MongoClient
from text_summary import Text_summary
from outline_config import OutlineConfig
from extract_index import Extract_Index

TYPE_ENTITY = 1
conf = OutlineConfig()
#  精读文本抽取规则
class Outline:
    def __init__(self, sentence, src_id, company_name, title, end_date='2016-12-31'):
        # 文章内容
        self.sentence = sentence
        # 公司名称
        self.company_name = company_name
        self.src_id = src_id
        self.title = title
        self.end_date = end_date
        # 模板名称
        self.template_name = company_name+title
        # 经营概况
        self.jygk_text = ''
        # 营业收入构成
        self.income_text = ''
        # 费用说明
        self.cost_text = ''
        # 核心竞争力分析
        self.competevie_text = ''
        # 未来发展展望
        self.future_text = ''
        # 风险提示
        self.risk_text = ''
        # 经营概况mongo追加指标
        self.jygk_index = ''


    # 加载配置文件
    def load_config(self, template_name):
        pass

    # 获取mysql数据库连接
    def getMysqlConn(self):
        host, port, dbname, user_name, password = conf.get_index_mysql_conn()
        conn = pymysql.connect(
            host=host,
            port=int(port),
            user=user_name,
            passwd=password,
            db=dbname,
            charset='utf8',
        )
        return conn

    # 获取mongodb访问连接
    def getMongoClient(self):
        dsn = conf.get_mongodb_conn()
        client = MongoClient(dsn)
        db = client.cr_data
        return db

    # 指标数据抽取
    def extract_index(self):
        mysql_conn = self.getMysqlConn()
        cur = mysql_conn.cursor()
        com_uni_code = self.get_company_unique_code()
        end_date = self.end_date
        # 指标数据查询
        sql = """SELECT t.sec_code,t1.end_date,t.com_uni_code,parent_netprofit,parent_netprofit_rate,overall_income,overall_income_rate,basic_perstock_income,basic_perstock_income_rate,
              t1.total_asset,sale_cost,sale_cost_rate,manage_cost,manage_cost_rate,fin_cost,fin_cost_rate,reduce_tax,reduce_tax_rate
              FROM sec_basic_info t  
              INNER JOIN
              (
              SELECT a.com_uni_code,a.end_date,a.parent_netprofit,c.parent_netprofit as parent_netprofit_rate,
              a.overall_income,c.overall_income as overall_income_rate,a.basic_perstock_income,c.basic_perstock_income as basic_perstock_income_rate,
              b.total_asset,a.sale_cost,c.sale_cost as sale_cost_rate,a.manage_cost,c.manage_cost as manage_cost_rate,a.fin_cost,c.fin_cost as fin_cost_rate,
              a.reduce_tax,c.reduce_tax as reduce_tax_rate
              FROM com_profit a
              INNER JOIN com_balance b  ON a.com_uni_code=b.com_uni_code 
              INNER JOIN com_profit_sheet_yoy c on b.com_uni_code = c.com_uni_code
              WHERE a.consolidation='1501002' AND a.end_date='%s'
              AND a.com_uni_code='%s' AND a.end_date=b.end_date  AND a.consolidation=b.consolidation and a.consolidation=c.consolidation AND a.end_date=c.end_date
              ) t1
              ON t.com_uni_code=t1.com_uni_code
              """ %(end_date, com_uni_code)
        cur.execute(sql)
        # 获取查询结果
        cwresults = cur.fetchall()
        if len(cwresults) == 0:
            return False
        # print(cwresults)
        # 产品收入
        sql = """
              SELECT
                  b.system_name,
                  a.*
              FROM
                  com_sale_structure a,
                  system_const b
              WHERE
                  b.system_uni_code = a.project_type
              AND end_date = '2016-12-31' and system_name in ('按行业','按产品')
              AND com_uni_code = 1579
              ORDER BY a.id
              """
        cur.execute(sql)
        results = cur.fetchall()
        # print "jieguo count:".count(results)
        cur.close()
        mysql_conn.close()
        # 组织语句
        # 经营概况
        self.jygk_text = self.template_name + '归属于母公司股东的净利润' + str(cwresults[0][3]) + "元,，同比增长" + str(cwresults[0][4] * 100) + "%，营业总收入" + str(cwresults[0][5]) + "元，同比增长" + str(
            cwresults[0][6] * 100) + "%，基本每股收益" + str(cwresults[0][7]) + "元，同比增长" + str(cwresults[0][8] * 100) + '%,总资产' + str(cwresults[0][9]) + '元'
        # print(self.jygk_text)
        # 费用说明
        self.cost_text = '2016年' + '销售费用' + str(cwresults[0][10]) + '元，同比增减' + str(
            cwresults[0][11] * 100) + '%，管理费用' + str(cwresults[0][12]) + '元，同比增减' + str(
            cwresults[0][13] * 100) + '%，财务费用' + str(cwresults[0][14]) + '元,同比增减' + str(
            cwresults[0][15] * 100) + '%，所得税费' + str(cwresults[0][16]) + '元，占比' + str(cwresults[0][17] * 100) + '%'
        # print(self.cost_text)
        if len(results) == 0:
            return False
        # 营业收入构成
        self.income_text = '2016年'
        for item in results:
            if item[10] == None:
                continue
            self.income_text += ' ' + str(item[6]) + str(item[7]) + '元，营业收入占比' + str(item[8] * 100) + '%，营业成本' + str(
                item[9]) + '，营业成本占比' + str(item[10] * 100) + '%，营业毛利' + str(item[11]) + '，营业毛利率' + str(item[12] * 100) + '%'
        # print(self.income_text)


    def business_index(self):
        extract_util = Extract_Index(self.sentence)
        self.jygk_index = extract_util.load_article()

    # 根据模板名称加载行业模板数据抽取规则
    def load_template(self):
        self.extract_index()
        self.business_index()
        self.exec_extract_rule()
        # print(self.jygk_text)
        # print(self.income_text)
        # print(self.cost_text)
        # print(self.competevie_text)
        # print(self.future_text)
        # print(self.risk_text)

    # 执行抽取规则
    def exec_extract_rule(self):
        # 计算机行业
        # 1、先抽取一整块内容
        soup = BeautifulSoup(self.sentence, 'lxml')
        # 删除outline
        if hasattr(soup, 'div') and soup.div is not None:
            soup.div['class'] = 'outline'
            soup.div.extract()
        # 获取原始文档，剔除不需要的内容
        origin_text = soup.get_text()
        rule = r'核心竞争力分析(.*)第五节'
        if len(re.findall(rule, origin_text, re.S)) > 0:
            origin_text = re.findall(rule, origin_text, re.S)[0]

        # 2、做数据清洗
        origin_text = origin_text.replace('□ 适用 √ 不适用', '').replace("√  □", "").replace('单位：万元', '').replace(
            '无', '').replace('否', '').replace('单位：元', '').replace('公司是需要遵守特殊行业的披露要求', '').replace('不适用', '').replace('适用', '')
        items = origin_text.split('\n')
        # 2.1 剔除一行 只有数字的
        # 2.2 剔除一行 只有3个字以内的
        delArr = []
        for item in items:
            if re.match(r"d+$", item.strip()):
                delArr.append(item)
                continue
            if len(item.strip()) <= 2:
                delArr.append(item)
                continue
        for d in delArr:
            items.remove(d)
        origin_text = '\n'.join(items)
        # 3、数据抽取

        # 核心竞争力分析
        competevie_rule = r'(.*)第四节'
        competevie_text = re.findall(competevie_rule, origin_text, re.S)
        if len(competevie_text) > 0:
            text_summary = Text_summary()
            dict = text_summary.summarize(competevie_text[0])
            for sent in dict['top_n_summary']:
                # print(sent.replace('\n', ''))
                self.competevie_text += sent.replace('\n', '')
        self.competevie_text = self.dealOrder(self.competevie_text)
        # print("===============================================================================")

        # 未来发展展望 公司关于公司未来发展的讨论与分析
        future_rule = r'公司未来发展的(.*)可能面对的风险'
        future_text = re.findall(future_rule, origin_text, re.S)
        if len(future_text) == 0:
            future_rule = r'公司未来发展的展望(.*)'
            future_text = re.findall(future_rule, origin_text, re.S)
        if len(future_text) > 0:
            text_summary = Text_summary()
            dict = text_summary.summarize(future_text[0])
            for sent in dict['mean_scored_summary']:
                # print(sent.replace('\n', ''))
                self.future_text += sent.replace('\n', '')
        self.future_text = self.dealOrder(self.future_text)
        # print("===============================================================================")

        # 公司面对的风险
        risk_rule = r'风险和应对措施(.*)'
        risk_text = re.findall(risk_rule, origin_text, re.S)
        if len(risk_text) == 0:
            risk_rule = r'可能面对的风险(.*)'
            risk_text = re.findall(risk_rule, origin_text, re.S)
        if len(risk_text) > 0:
            dict = text_summary.summarize(risk_text[0])
            for sent in dict['mean_scored_summary']:
                self.risk_text += sent.replace('\n', '')
                # print(sent.replace('\n', ''))
        self.risk_text = self.dealOrder(self.risk_text)
        # print("===============================================================================")
        # print(self.risk_text)
        # print(risk_text)
        # print(origin_text)
        # html = soup.find_all(text=re.compile("核心竞争力分析"))
        # print(html)

    # 处理文本序号问题
    def dealOrder(self, sentence):
        # 序号规则一  （一）
        # 序号规则二  （1）
        # 序号规则三   1、
        # 序号规则四   1）、（.{1}）
        rules = [r'[(,（]{1}.{1}[),）]{1}', r'\d{1}[、,.]{1}',
                 r'\d{1}\)[、,.]{1}', r'[第]{1}.{1}[段]{0,1}']
        results = []
        for rule in rules:
            results = re.findall(rule, sentence, re.S)
            if len(results) == 1:
                sentence = sentence.replace(results[0], '')
            if len(results) > 1:
                break
        # 替换
        for index, item in enumerate(results):
            sentence = sentence.replace(item, str(index + 1) + '、')
        return sentence

    # 合并数据规则
    def combine_data(self):
        pass

    # 保存到数据库
    def saveToData(self, file_name):
        pass

    # 保存到本地文件
    def saveToText(self):
        file_object = open(self.template_name+'精读版.txt', 'w', encoding='utf-8')
        article_str = self.constructTemplate()
        print(article_str)
        # article_str = article_str.encode(encoding='gbk')
        file_object.writelines(article_str)
        file_object.close()

    def get_company_unique_code(self):
        src_id = self.src_id
        mysql_conn = getDevMysqlConn()
        cur = mysql_conn.cursor()
        sqlStr = '''select c.com_uni_code from notice a
                left join company_basicinfo b on a.stockcode = b.stock_code
                left JOIN sec_basic_info c on b.wind_code=c.abc_code
                where a.src_id='%s' '''
        sql = sqlStr % src_id
        cur.execute(sql)
        results = cur.fetchone()
        if len(results) > 0:
            return results[0]

    def get_notice_id_by_srcid(self):
        src_id = self.src_id
        mysql_conn = getDevMysqlConn()
        cur = mysql_conn.cursor()
        sql = '''select b.id from kg_notice_basic b 
                   LEFT JOIN notice n on b.article_id = n.added_id
                   where n.src_id = '%s' ''' % src_id
        cur.execute(sql)
        results = cur.fetchone()
        if len(results) > 0:
            return results[0]

    def constructTemplate(self):
        if self.jygk_index is not None:
            self.jygk_text += self.jygk_index
        tecTemplate = Template('<h1>1、经营情况概述</h1> \n '
                               '<p>$jygk_text</p> \n'
                               '<h1>2、营业收入构成</h1> \n'
                               '<p>$income_text</p> \n'
                               '<h1>3、费用与研发投入说明</h1> \n'
                               '<p>$cost_text</p> \n'
                               '<h1>4、核心竞争优势</h1> \n'
                               '<p>$competevie_text</p> \n'
                               '<h1>5、未来发展展望</h1> \n'
                               '<p>$future_text</p> \n'
                               '<h1>6、风险提示</h1> \n'
                               '<p>$risk_text</p> \n')
        d = dict(jygk_text=self.jygk_text,
                 income_text=self.income_text,
                 cost_text=self.cost_text,
                 competevie_text=self.competevie_text,
                 future_text=self.future_text,
                 risk_text=self.risk_text)
        return tecTemplate.safe_substitute(d)

    def getAllByArticle(self, article_id):
        if article_id is None or article_id == '':
            self.errMessage = '文章id有误'
            return False
        mysql_conn = getDevMysqlConn()
        cur = mysql_conn.cursor()
        sql = '''SELECT c.company_name FROM notice n 
                  INNER JOIN company_basicinfo c ON n.stockcode=c.stock_code 
                  WHERE added_id = '%s.' LIMIT 1 ''' % article_id
        cur.execute(sql)
        basic_com_name = cur.fetchone()
        if len(basic_com_name) == 0:
            self.errMessage = '公告公司有误'
            return False
        sql = '''Select * from kg_notice_basic where article_id='%s' '''% article_id
        cur.execute(sql)
        notice = cur.fetchone()
        if len(notice) == 0:
            self.errMessage = '公告有误'
            return False
        # notice[0] is notice id
        notice_id = notice[0]
        sql = '''Select * from kg_notice_related 
                  where notice_id='%s' and subject_type=%s ''' % (notice_id, TYPE_ENTITY)
        cur.execute(sql)
        entity_relateds = cur.fetchall()
        entity_ids = []
        if len(entity_relateds):
            entity_ids = array_column(entity_relateds,2)
        sql = '''Select * from kg_entity 
                  WHERE id IN (%s)''' % entity_ids
        cur.execute(sql)
        entity_relateds = cur.fetchall()


# 取得查询结果集的固定列
def array_column(arrs,index):
    newArrs = []
    for a in arrs:
        if len(a)>index:
            newArrs.append(a[index])
    return newArrs

def write(file_name):
    obj = divide_file_name(get_file_name(file_name))
    root = conf.get_project_path()
    file = open(root+file_name, 'r', encoding='utf-8')
    lines = file.read()
    outlie_util = Outline(lines, obj['src_id'], obj['company_name'], obj['year'])
    outlie_util.load_template()
    outlie_util.saveToText()

def update_outline(file_name):
    obj = divide_file_name(get_file_name(file_name))
    root = conf.get_project_path()
    file = open(root + file_name, 'r', encoding='utf-8')
    lines = file.read()
    outlie_util = Outline(lines, obj['src_id'], obj['company_name'], obj['year'])
    outlie_util.load_template()
    mysql_conn = getDevMysqlConn()
    cur = mysql_conn.cursor()
    title = outlie_util.template_name + '精读版'
    outline = outlie_util.constructTemplate()
    notice_id = outlie_util.get_notice_id_by_srcid()
    notice_time = outlie_util.title
    update_sql = '''insert INTO kg_outline (title,outline,notice_id,notice_time,isrobot) 
                    VALUES ('%s','%s','%s','%s',%s); ''' % (title, outline, notice_id, notice_time, 1)
    cur.execute(update_sql)
    mysql_conn.commit()
    cur.close()
    mysql_conn.close()

# 获取开发mysql数据库连接
def getDevMysqlConn():
    host, port, dbname, user_name, password = conf.get_mysql_dev_conn()
    conn = pymysql.connect(
        host=host,
        port=int(port),
        user=user_name,
        passwd=password,
        db=dbname,
        charset='utf8',
    )
    return conn

# 获取不带文件类型的文件名称
def get_file_name(file):
    files = os.path.splitext(file)
    if len(files) > 0:
        return files[0]

def divide_file_name(file_name):
    files = file_name.split('_')
    return {
        'src_id':files[0]+"_"+files[1],
        'company_name':files[2],
        'year':files[3]
    }
def main():
    path = conf.get_project_path()
    dir_list = os.listdir(path)
    file_list = []
    for file in dir_list:
        if os.path.isfile(path + file):
            file_list.append(file)
    futures = set()
    with ThreadPoolExecutor(1) as executor:
        # executor.map(write, text_list)
        for f in file_list:
            future = executor.submit(update_outline, f)
            futures.add(future)
    try:
        for future in as_completed(futures):
            err = future.exception()
            if err is not None:
                raise err
    except KeyboardInterrupt:
        print("stopped by hand")


if __name__ == "__main__":
    main()
    # outline = Outline(1,1,1,1)
    # outline.getAllByArticle('59')
    # outlie_util.exec_extract_rule();
